{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AB155B13302D459CBAAABA327849B76D",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "# 说明\n",
    "特征工程部分照搬社区的baseline未作改动，仅仅进行了模型的部分参数调整和基于stacking的模型融合。\n",
    "另外，由于是输出概率，后续按照回归去做，故删除了不平衡样本的处理，一开始当成分类去做，最高只能到0.8+，按照回归轻松0.9+\n",
    "参数仅做了简单的调整，非最优，线下0.9361018703876826，线上验证0.93536922"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "C325760DBE164B398588C602C8C376BC",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "# 查看数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "id": "0A028E17E34B4CFEABE9DFF956D89741",
    "jupyter": {
     "outputs_hidden": false
    },
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A module that was compiled using NumPy 1.x cannot be run in\n",
      "NumPy 2.2.6 as it may crash. To support both 1.x and 2.x\n",
      "versions of NumPy, modules must be compiled with NumPy 2.0.\n",
      "Some module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n",
      "\n",
      "If you are a user of the module, the easiest solution will be to\n",
      "downgrade to 'numpy<2' or try to upgrade the affected module.\n",
      "We expect that some modules will need time to support NumPy 2.\n",
      "\n",
      "Traceback (most recent call last):  File \"<frozen runpy>\", line 198, in _run_module_as_main\n",
      "  File \"<frozen runpy>\", line 88, in _run_code\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel_launcher.py\", line 17, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\traitlets\\config\\application.py\", line 1075, in launch_instance\n",
      "    app.start()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelapp.py\", line 701, in start\n",
      "    self.io_loop.start()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\tornado\\platform\\asyncio.py\", line 205, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\windows_events.py\", line 322, in run_forever\n",
      "    super().run_forever()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\base_events.py\", line 641, in run_forever\n",
      "    self._run_once()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\base_events.py\", line 1987, in _run_once\n",
      "    handle._run()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\events.py\", line 88, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in dispatch_queue\n",
      "    await self.process_one()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 523, in process_one\n",
      "    await dispatch(*args)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 429, in dispatch_shell\n",
      "    await result\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 767, in execute_request\n",
      "    reply_content = await reply_content\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\ipkernel.py\", line 429, in do_execute\n",
      "    res = shell.run_cell(\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell\n",
      "    return super().run_cell(*args, **kwargs)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3075, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3130, in _run_cell\n",
      "    result = runner(coro)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3334, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3517, in run_ast_nodes\n",
      "    if await self.run_code(code, result, async_=asy):\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3577, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\zh410st005\\AppData\\Local\\Temp\\ipykernel_10864\\762988937.py\", line 1, in <module>\n",
      "    import pandas as pd\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\pandas\\__init__.py\", line 26, in <module>\n",
      "    from pandas.compat import (\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\pandas\\compat\\__init__.py\", line 27, in <module>\n",
      "    from pandas.compat.pyarrow import (\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\pandas\\compat\\pyarrow.py\", line 8, in <module>\n",
      "    import pyarrow as pa\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\pyarrow\\__init__.py\", line 65, in <module>\n",
      "    import pyarrow.lib as _lib\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "\nA module that was compiled using NumPy 1.x cannot be run in\nNumPy 2.2.6 as it may crash. To support both 1.x and 2.x\nversions of NumPy, modules must be compiled with NumPy 2.0.\nSome module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n\nIf you are a user of the module, the easiest solution will be to\ndowngrade to 'numpy<2' or try to upgrade the affected module.\nWe expect that some modules will need time to support NumPy 2.\n\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python312\\site-packages\\numpy\\core\\_multiarray_umath.py:44\u001b[0m, in \u001b[0;36m__getattr__\u001b[1;34m(attr_name)\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[38;5;66;03m# Also print the message (with traceback).  This is because old versions\u001b[39;00m\n\u001b[0;32m     40\u001b[0m     \u001b[38;5;66;03m# of NumPy unfortunately set up the import to replace (and hide) the\u001b[39;00m\n\u001b[0;32m     41\u001b[0m     \u001b[38;5;66;03m# error.  The traceback shouldn't be needed, but e.g. pytest plugins\u001b[39;00m\n\u001b[0;32m     42\u001b[0m     \u001b[38;5;66;03m# seem to swallow it and we should be failing anyway...\u001b[39;00m\n\u001b[0;32m     43\u001b[0m     sys\u001b[38;5;241m.\u001b[39mstderr\u001b[38;5;241m.\u001b[39mwrite(msg \u001b[38;5;241m+\u001b[39m tb_msg)\n\u001b[1;32m---> 44\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(msg)\n\u001b[0;32m     46\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(_multiarray_umath, attr_name, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m     47\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m ret \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[1;31mImportError\u001b[0m: \nA module that was compiled using NumPy 1.x cannot be run in\nNumPy 2.2.6 as it may crash. To support both 1.x and 2.x\nversions of NumPy, modules must be compiled with NumPy 2.0.\nSome module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n\nIf you are a user of the module, the easiest solution will be to\ndowngrade to 'numpy<2' or try to upgrade the affected module.\nWe expect that some modules will need time to support NumPy 2.\n\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A module that was compiled using NumPy 1.x cannot be run in\n",
      "NumPy 2.2.6 as it may crash. To support both 1.x and 2.x\n",
      "versions of NumPy, modules must be compiled with NumPy 2.0.\n",
      "Some module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n",
      "\n",
      "If you are a user of the module, the easiest solution will be to\n",
      "downgrade to 'numpy<2' or try to upgrade the affected module.\n",
      "We expect that some modules will need time to support NumPy 2.\n",
      "\n",
      "Traceback (most recent call last):  File \"<frozen runpy>\", line 198, in _run_module_as_main\n",
      "  File \"<frozen runpy>\", line 88, in _run_code\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel_launcher.py\", line 17, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\traitlets\\config\\application.py\", line 1075, in launch_instance\n",
      "    app.start()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelapp.py\", line 701, in start\n",
      "    self.io_loop.start()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\tornado\\platform\\asyncio.py\", line 205, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\windows_events.py\", line 322, in run_forever\n",
      "    super().run_forever()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\base_events.py\", line 641, in run_forever\n",
      "    self._run_once()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\base_events.py\", line 1987, in _run_once\n",
      "    handle._run()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\events.py\", line 88, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in dispatch_queue\n",
      "    await self.process_one()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 523, in process_one\n",
      "    await dispatch(*args)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 429, in dispatch_shell\n",
      "    await result\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 767, in execute_request\n",
      "    reply_content = await reply_content\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\ipkernel.py\", line 429, in do_execute\n",
      "    res = shell.run_cell(\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell\n",
      "    return super().run_cell(*args, **kwargs)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3075, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3130, in _run_cell\n",
      "    result = runner(coro)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3334, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3517, in run_ast_nodes\n",
      "    if await self.run_code(code, result, async_=asy):\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3577, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\zh410st005\\AppData\\Local\\Temp\\ipykernel_10864\\762988937.py\", line 1, in <module>\n",
      "    import pandas as pd\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\pandas\\__init__.py\", line 49, in <module>\n",
      "    from pandas.core.api import (\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\pandas\\core\\api.py\", line 1, in <module>\n",
      "    from pandas._libs import (\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\pandas\\_libs\\__init__.py\", line 17, in <module>\n",
      "    import pandas._libs.pandas_datetime  # noqa: F401 # isort: skip # type: ignore[reportUnusedImport]\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "\nA module that was compiled using NumPy 1.x cannot be run in\nNumPy 2.2.6 as it may crash. To support both 1.x and 2.x\nversions of NumPy, modules must be compiled with NumPy 2.0.\nSome module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n\nIf you are a user of the module, the easiest solution will be to\ndowngrade to 'numpy<2' or try to upgrade the affected module.\nWe expect that some modules will need time to support NumPy 2.\n\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python312\\site-packages\\numpy\\core\\_multiarray_umath.py:44\u001b[0m, in \u001b[0;36m__getattr__\u001b[1;34m(attr_name)\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[38;5;66;03m# Also print the message (with traceback).  This is because old versions\u001b[39;00m\n\u001b[0;32m     40\u001b[0m     \u001b[38;5;66;03m# of NumPy unfortunately set up the import to replace (and hide) the\u001b[39;00m\n\u001b[0;32m     41\u001b[0m     \u001b[38;5;66;03m# error.  The traceback shouldn't be needed, but e.g. pytest plugins\u001b[39;00m\n\u001b[0;32m     42\u001b[0m     \u001b[38;5;66;03m# seem to swallow it and we should be failing anyway...\u001b[39;00m\n\u001b[0;32m     43\u001b[0m     sys\u001b[38;5;241m.\u001b[39mstderr\u001b[38;5;241m.\u001b[39mwrite(msg \u001b[38;5;241m+\u001b[39m tb_msg)\n\u001b[1;32m---> 44\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(msg)\n\u001b[0;32m     46\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(_multiarray_umath, attr_name, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m     47\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m ret \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[1;31mImportError\u001b[0m: \nA module that was compiled using NumPy 1.x cannot be run in\nNumPy 2.2.6 as it may crash. To support both 1.x and 2.x\nversions of NumPy, modules must be compiled with NumPy 2.0.\nSome module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n\nIf you are a user of the module, the easiest solution will be to\ndowngrade to 'numpy<2' or try to upgrade the affected module.\nWe expect that some modules will need time to support NumPy 2.\n\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "numpy.core.multiarray failed to import",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\pandas\\__init__.py:49\u001b[0m\n\u001b[0;32m     46\u001b[0m \u001b[38;5;66;03m# let init-time option registration happen\u001b[39;00m\n\u001b[0;32m     47\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig_init\u001b[39;00m  \u001b[38;5;66;03m# pyright: ignore[reportUnusedImport] # noqa: F401\u001b[39;00m\n\u001b[1;32m---> 49\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapi\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     50\u001b[0m     \u001b[38;5;66;03m# dtype\u001b[39;00m\n\u001b[0;32m     51\u001b[0m     ArrowDtype,\n\u001b[0;32m     52\u001b[0m     Int8Dtype,\n\u001b[0;32m     53\u001b[0m     Int16Dtype,\n\u001b[0;32m     54\u001b[0m     Int32Dtype,\n\u001b[0;32m     55\u001b[0m     Int64Dtype,\n\u001b[0;32m     56\u001b[0m     UInt8Dtype,\n\u001b[0;32m     57\u001b[0m     UInt16Dtype,\n\u001b[0;32m     58\u001b[0m     UInt32Dtype,\n\u001b[0;32m     59\u001b[0m     UInt64Dtype,\n\u001b[0;32m     60\u001b[0m     Float32Dtype,\n\u001b[0;32m     61\u001b[0m     Float64Dtype,\n\u001b[0;32m     62\u001b[0m     CategoricalDtype,\n\u001b[0;32m     63\u001b[0m     PeriodDtype,\n\u001b[0;32m     64\u001b[0m     IntervalDtype,\n\u001b[0;32m     65\u001b[0m     DatetimeTZDtype,\n\u001b[0;32m     66\u001b[0m     StringDtype,\n\u001b[0;32m     67\u001b[0m     BooleanDtype,\n\u001b[0;32m     68\u001b[0m     \u001b[38;5;66;03m# missing\u001b[39;00m\n\u001b[0;32m     69\u001b[0m     NA,\n\u001b[0;32m     70\u001b[0m     isna,\n\u001b[0;32m     71\u001b[0m     isnull,\n\u001b[0;32m     72\u001b[0m     notna,\n\u001b[0;32m     73\u001b[0m     notnull,\n\u001b[0;32m     74\u001b[0m     \u001b[38;5;66;03m# indexes\u001b[39;00m\n\u001b[0;32m     75\u001b[0m     Index,\n\u001b[0;32m     76\u001b[0m     CategoricalIndex,\n\u001b[0;32m     77\u001b[0m     RangeIndex,\n\u001b[0;32m     78\u001b[0m     MultiIndex,\n\u001b[0;32m     79\u001b[0m     IntervalIndex,\n\u001b[0;32m     80\u001b[0m     TimedeltaIndex,\n\u001b[0;32m     81\u001b[0m     DatetimeIndex,\n\u001b[0;32m     82\u001b[0m     PeriodIndex,\n\u001b[0;32m     83\u001b[0m     IndexSlice,\n\u001b[0;32m     84\u001b[0m     \u001b[38;5;66;03m# tseries\u001b[39;00m\n\u001b[0;32m     85\u001b[0m     NaT,\n\u001b[0;32m     86\u001b[0m     Period,\n\u001b[0;32m     87\u001b[0m     period_range,\n\u001b[0;32m     88\u001b[0m     Timedelta,\n\u001b[0;32m     89\u001b[0m     timedelta_range,\n\u001b[0;32m     90\u001b[0m     Timestamp,\n\u001b[0;32m     91\u001b[0m     date_range,\n\u001b[0;32m     92\u001b[0m     bdate_range,\n\u001b[0;32m     93\u001b[0m     Interval,\n\u001b[0;32m     94\u001b[0m     interval_range,\n\u001b[0;32m     95\u001b[0m     DateOffset,\n\u001b[0;32m     96\u001b[0m     \u001b[38;5;66;03m# conversion\u001b[39;00m\n\u001b[0;32m     97\u001b[0m     to_numeric,\n\u001b[0;32m     98\u001b[0m     to_datetime,\n\u001b[0;32m     99\u001b[0m     to_timedelta,\n\u001b[0;32m    100\u001b[0m     \u001b[38;5;66;03m# misc\u001b[39;00m\n\u001b[0;32m    101\u001b[0m     Flags,\n\u001b[0;32m    102\u001b[0m     Grouper,\n\u001b[0;32m    103\u001b[0m     factorize,\n\u001b[0;32m    104\u001b[0m     unique,\n\u001b[0;32m    105\u001b[0m     value_counts,\n\u001b[0;32m    106\u001b[0m     NamedAgg,\n\u001b[0;32m    107\u001b[0m     array,\n\u001b[0;32m    108\u001b[0m     Categorical,\n\u001b[0;32m    109\u001b[0m     set_eng_float_format,\n\u001b[0;32m    110\u001b[0m     Series,\n\u001b[0;32m    111\u001b[0m     DataFrame,\n\u001b[0;32m    112\u001b[0m )\n\u001b[0;32m    114\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdtypes\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdtypes\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SparseDtype\n\u001b[0;32m    116\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtseries\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapi\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m infer_freq\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\pandas\\core\\api.py:1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_libs\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m      2\u001b[0m     NaT,\n\u001b[0;32m      3\u001b[0m     Period,\n\u001b[0;32m      4\u001b[0m     Timedelta,\n\u001b[0;32m      5\u001b[0m     Timestamp,\n\u001b[0;32m      6\u001b[0m )\n\u001b[0;32m      7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_libs\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmissing\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m NA\n\u001b[0;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdtypes\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdtypes\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     10\u001b[0m     ArrowDtype,\n\u001b[0;32m     11\u001b[0m     CategoricalDtype,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     14\u001b[0m     PeriodDtype,\n\u001b[0;32m     15\u001b[0m )\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\pandas\\_libs\\__init__.py:17\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;66;03m# Below imports needs to happen first to ensure pandas top level\u001b[39;00m\n\u001b[0;32m     14\u001b[0m \u001b[38;5;66;03m# module gets monkeypatched with the pandas_datetime_CAPI\u001b[39;00m\n\u001b[0;32m     15\u001b[0m \u001b[38;5;66;03m# see pandas_datetime_exec in pd_datetime.c\u001b[39;00m\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_libs\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpandas_parser\u001b[39;00m  \u001b[38;5;66;03m# isort: skip # type: ignore[reportUnusedImport]\u001b[39;00m\n\u001b[1;32m---> 17\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_libs\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpandas_datetime\u001b[39;00m  \u001b[38;5;66;03m# noqa: F401 # isort: skip # type: ignore[reportUnusedImport]\u001b[39;00m\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_libs\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01minterval\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Interval\n\u001b[0;32m     19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_libs\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtslibs\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     20\u001b[0m     NaT,\n\u001b[0;32m     21\u001b[0m     NaTType,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     26\u001b[0m     iNaT,\n\u001b[0;32m     27\u001b[0m )\n",
      "\u001b[1;31mImportError\u001b[0m: numpy.core.multiarray failed to import"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "plt.rcParams['axes.unicode_minus'] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "id": "001F87A7D41644E4BA2EA25513B0D1B3",
    "jupyter": {
     "outputs_hidden": false
    },
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train=pd.read_csv(r'/home/mw/input/data9803/train_set.csv')\n",
    "test=pd.read_csv('/home/mw/input/data9803/test_set.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "id": "B8C543716B964973872549E7B4DFDFAA",
    "jupyter": {
     "outputs_hidden": false
    },
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = pd.concat([train.drop(['y'],axis=1),test],axis=0).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "id": "D7CE94A95A2544AE808C26A4B1901BA7",
    "jupyter": {
     "outputs_hidden": false
    },
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 对object型数据查看unique\n",
    "str_features = []\n",
    "num_features=[]\n",
    "for col in train.columns:\n",
    "    if train[col].dtype=='object':\n",
    "        str_features.append(col)\n",
    "        print(col,':  ',train[col].unique())\n",
    "    if train[col].dtype=='int64' and col not in ['ID','y']:\n",
    "        num_features.append(col)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "29CB9E135078402DBE2F479858631C1D",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "# 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "928046C510F54165A50D2FFBEFB4A5B5",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from scipy.stats import chi2_contingency       # 数值型特征检验，检验特征与标签的关系\n",
    "from scipy.stats import f_oneway,ttest_ind     # 分类型特征检验，检验特征与标签的关系"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "B5098300108B4CB784E1E1D8A1A7AF40",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#----------数据集处理--------------#\n",
    "from sklearn.model_selection import train_test_split        # 划分训练集和验证集\n",
    "from sklearn.model_selection import KFold,StratifiedKFold   # k折交叉\n",
    "from imblearn.combine import SMOTETomek,SMOTEENN            # 综合采样\n",
    "from imblearn.over_sampling import SMOTE                    # 过采样\n",
    "from imblearn.under_sampling import RandomUnderSampler      # 欠采样\n",
    "\n",
    "#----------数据处理--------------#\n",
    "from sklearn.preprocessing import StandardScaler # 标准化\n",
    "from sklearn.preprocessing import OneHotEncoder  # 热独编码\n",
    "from sklearn.preprocessing import OrdinalEncoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AB4FDF679736489B86708802E2B4E05B",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 特征处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "B76FEED533744B93AD63ED6825716E5C",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "**连续变量即数值化数据做标准化处理**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DD0BE0C3EF0D4755A0F255565CA064B3",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 异常值处理\n",
    "def outlier_processing(dfx):\n",
    "    df = dfx.copy()\n",
    "    q1 = df.quantile(q=0.25)\n",
    "    q3 = df.quantile(q=0.75)\n",
    "    iqr = q3 - q1\n",
    "    Umin = q1 - 1.5*iqr\n",
    "    Umax = q3 + 1.5*iqr \n",
    "    df[df>Umax] = df[df<=Umax].max()\n",
    "    df[df<Umin] = df[df>=Umin].min()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "353D496D273C4456A8EF7C663D1C81C1",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train['age']=outlier_processing(train['age'])\n",
    "train['day']=outlier_processing(train['day'])\n",
    "train['duration']=outlier_processing(train['duration'])\n",
    "train['campaign']=outlier_processing(train['campaign'])\n",
    "\n",
    "\n",
    "test['age']=outlier_processing(test['age'])\n",
    "test['day']=outlier_processing(test['day'])\n",
    "test['duration']=outlier_processing(test['duration'])\n",
    "test['campaign']=outlier_processing(test['campaign'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "605C7188AA404524AA2B935E363D1E6C",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[2], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m train[num_features]\u001b[38;5;241m.\u001b[39mdescribe()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'train' is not defined"
     ]
    }
   ],
   "source": [
    "train[num_features].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FA30A072639C418283A5F9FF7BB70A5A",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "**分类变量做编码处理**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "B843C59639844E1A872AECAFE7926878",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[3], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m dummy_train\u001b[38;5;241m=\u001b[39mtrain\u001b[38;5;241m.\u001b[39mjoin(pd\u001b[38;5;241m.\u001b[39mget_dummies(train[str_features]))\u001b[38;5;241m.\u001b[39mdrop(str_features,axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mdrop([\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mID\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124my\u001b[39m\u001b[38;5;124m'\u001b[39m],axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m      2\u001b[0m dummy_test\u001b[38;5;241m=\u001b[39mtest\u001b[38;5;241m.\u001b[39mjoin(pd\u001b[38;5;241m.\u001b[39mget_dummies(test[str_features]))\u001b[38;5;241m.\u001b[39mdrop(str_features,axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mdrop([\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mID\u001b[39m\u001b[38;5;124m'\u001b[39m],axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'train' is not defined"
     ]
    }
   ],
   "source": [
    "dummy_train=train.join(pd.get_dummies(train[str_features])).drop(str_features,axis=1).drop(['ID','y'],axis=1)\n",
    "dummy_test=test.join(pd.get_dummies(test[str_features])).drop(str_features,axis=1).drop(['ID'],axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "F2EA03BA7FAA45FF85F2CEDA21AB2F25",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 统计检验与特征筛选 \n",
    "\n",
    "\n",
    "**连续变量-连续变量  相关分析**\n",
    "\n",
    "**连续变量-分类变量  T检验/方差分析**\n",
    "\n",
    "**分类变量-分类变量  卡方检验**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "500456AC13CA4815858E06D09EE17A9E",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "**对类别标签（离散变量）用卡方检验分析重要性**\n",
    "\n",
    "卡方检验认为显著水平大于95%是差异性显著的，这里即看p值是否是p>0.05，若p>0.05，则说明特征不会呈现差异性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "40798A95334C4A85812FCB8A3CE6DD75",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'str_features' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[4], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m col \u001b[38;5;129;01min\u001b[39;00m str_features:\n\u001b[0;32m      2\u001b[0m     obs\u001b[38;5;241m=\u001b[39mpd\u001b[38;5;241m.\u001b[39mcrosstab(train[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124my\u001b[39m\u001b[38;5;124m'\u001b[39m],\n\u001b[0;32m      3\u001b[0m                     train[col],\n\u001b[0;32m      4\u001b[0m                     rownames\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124my\u001b[39m\u001b[38;5;124m'\u001b[39m],\n\u001b[0;32m      5\u001b[0m                     colnames\u001b[38;5;241m=\u001b[39m[col])\n\u001b[0;32m      6\u001b[0m     chi2, p, dof, expect \u001b[38;5;241m=\u001b[39m chi2_contingency(obs)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'str_features' is not defined"
     ]
    }
   ],
   "source": [
    "for col in str_features:\n",
    "    obs=pd.crosstab(train['y'],\n",
    "                    train[col],\n",
    "                    rownames=['y'],\n",
    "                    colnames=[col])\n",
    "    chi2, p, dof, expect = chi2_contingency(obs)\n",
    "    print(\"{} 卡方检验p值: {:.4f}\".format(col,p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "30A24D4C89AF4BDAA0115FDF8F7C12E5",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "**对连续变量做方差分析进行特征筛选**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "43185B05C4AB48628437C4224F86F348",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A module that was compiled using NumPy 1.x cannot be run in\n",
      "NumPy 2.2.6 as it may crash. To support both 1.x and 2.x\n",
      "versions of NumPy, modules must be compiled with NumPy 2.0.\n",
      "Some module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n",
      "\n",
      "If you are a user of the module, the easiest solution will be to\n",
      "downgrade to 'numpy<2' or try to upgrade the affected module.\n",
      "We expect that some modules will need time to support NumPy 2.\n",
      "\n",
      "Traceback (most recent call last):  File \"<frozen runpy>\", line 198, in _run_module_as_main\n",
      "  File \"<frozen runpy>\", line 88, in _run_code\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel_launcher.py\", line 17, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\traitlets\\config\\application.py\", line 1075, in launch_instance\n",
      "    app.start()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelapp.py\", line 701, in start\n",
      "    self.io_loop.start()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\tornado\\platform\\asyncio.py\", line 205, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\windows_events.py\", line 322, in run_forever\n",
      "    super().run_forever()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\base_events.py\", line 641, in run_forever\n",
      "    self._run_once()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\base_events.py\", line 1987, in _run_once\n",
      "    handle._run()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\events.py\", line 88, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in dispatch_queue\n",
      "    await self.process_one()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 523, in process_one\n",
      "    await dispatch(*args)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 429, in dispatch_shell\n",
      "    await result\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 767, in execute_request\n",
      "    reply_content = await reply_content\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\ipkernel.py\", line 429, in do_execute\n",
      "    res = shell.run_cell(\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell\n",
      "    return super().run_cell(*args, **kwargs)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3075, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3130, in _run_cell\n",
      "    result = runner(coro)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3334, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3517, in run_ast_nodes\n",
      "    if await self.run_code(code, result, async_=asy):\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3577, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\zh410st005\\AppData\\Local\\Temp\\ipykernel_10864\\1636635545.py\", line 1, in <module>\n",
      "    from sklearn.feature_selection import SelectKBest,f_classif\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\__init__.py\", line 87, in <module>\n",
      "    from .base import clone\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\base.py\", line 19, in <module>\n",
      "    from .utils import _IS_32BIT\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\utils\\__init__.py\", line 16, in <module>\n",
      "    from scipy.sparse import issparse\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\__init__.py\", line 295, in <module>\n",
      "    from ._csr import *\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\_csr.py\", line 11, in <module>\n",
      "    from ._sparsetools import (csr_tocsc, csr_tobsr, csr_count_blocks,\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "\nA module that was compiled using NumPy 1.x cannot be run in\nNumPy 2.2.6 as it may crash. To support both 1.x and 2.x\nversions of NumPy, modules must be compiled with NumPy 2.0.\nSome module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n\nIf you are a user of the module, the easiest solution will be to\ndowngrade to 'numpy<2' or try to upgrade the affected module.\nWe expect that some modules will need time to support NumPy 2.\n\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python312\\site-packages\\numpy\\core\\_multiarray_umath.py:44\u001b[0m, in \u001b[0;36m__getattr__\u001b[1;34m(attr_name)\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[38;5;66;03m# Also print the message (with traceback).  This is because old versions\u001b[39;00m\n\u001b[0;32m     40\u001b[0m     \u001b[38;5;66;03m# of NumPy unfortunately set up the import to replace (and hide) the\u001b[39;00m\n\u001b[0;32m     41\u001b[0m     \u001b[38;5;66;03m# error.  The traceback shouldn't be needed, but e.g. pytest plugins\u001b[39;00m\n\u001b[0;32m     42\u001b[0m     \u001b[38;5;66;03m# seem to swallow it and we should be failing anyway...\u001b[39;00m\n\u001b[0;32m     43\u001b[0m     sys\u001b[38;5;241m.\u001b[39mstderr\u001b[38;5;241m.\u001b[39mwrite(msg \u001b[38;5;241m+\u001b[39m tb_msg)\n\u001b[1;32m---> 44\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(msg)\n\u001b[0;32m     46\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(_multiarray_umath, attr_name, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m     47\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m ret \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[1;31mImportError\u001b[0m: \nA module that was compiled using NumPy 1.x cannot be run in\nNumPy 2.2.6 as it may crash. To support both 1.x and 2.x\nversions of NumPy, modules must be compiled with NumPy 2.0.\nSome module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n\nIf you are a user of the module, the easiest solution will be to\ndowngrade to 'numpy<2' or try to upgrade the affected module.\nWe expect that some modules will need time to support NumPy 2.\n\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "numpy.core.multiarray failed to import",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[5], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msklearn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mfeature_selection\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SelectKBest,f_classif\n\u001b[0;32m      3\u001b[0m f,p\u001b[38;5;241m=\u001b[39mf_classif(train[num_features],train[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124my\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[0;32m      4\u001b[0m k \u001b[38;5;241m=\u001b[39m f\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m-\u001b[39m (p \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0.05\u001b[39m)\u001b[38;5;241m.\u001b[39msum()\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\__init__.py:87\u001b[0m\n\u001b[0;32m     73\u001b[0m     \u001b[38;5;66;03m# We are not importing the rest of scikit-learn during the build\u001b[39;00m\n\u001b[0;32m     74\u001b[0m     \u001b[38;5;66;03m# process, as it may not be compiled yet\u001b[39;00m\n\u001b[0;32m     75\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     81\u001b[0m     \u001b[38;5;66;03m# later is linked to the OpenMP runtime to make it possible to introspect\u001b[39;00m\n\u001b[0;32m     82\u001b[0m     \u001b[38;5;66;03m# it and importing it first would fail if the OpenMP dll cannot be found.\u001b[39;00m\n\u001b[0;32m     83\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     84\u001b[0m         __check_build,  \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[0;32m     85\u001b[0m         _distributor_init,  \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[0;32m     86\u001b[0m     )\n\u001b[1;32m---> 87\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbase\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m clone\n\u001b[0;32m     88\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_show_versions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m show_versions\n\u001b[0;32m     90\u001b[0m     __all__ \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m     91\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcalibration\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     92\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcluster\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    133\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mshow_versions\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m    134\u001b[0m     ]\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\base.py:19\u001b[0m\n\u001b[0;32m     17\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m config_context, get_config\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m InconsistentVersionWarning\n\u001b[1;32m---> 19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _IS_32BIT\n\u001b[0;32m     20\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_estimator_html_repr\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _HTMLDocumentationLinkMixin, estimator_html_repr\n\u001b[0;32m     21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_metadata_requests\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _MetadataRequester, _routing_enabled\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\utils\\__init__.py:16\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mitertools\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m compress, islice\n\u001b[0;32m     15\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m---> 16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msparse\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m issparse\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m get_config\n\u001b[0;32m     19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DataConversionWarning\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\__init__.py:295\u001b[0m\n\u001b[0;32m    292\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwarnings\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01m_warnings\u001b[39;00m\n\u001b[0;32m    294\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_base\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m--> 295\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_csr\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[0;32m    296\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_csc\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[0;32m    297\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_lil\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\_csr.py:11\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_matrix\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m spmatrix\n\u001b[0;32m     10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_base\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _spbase, sparray\n\u001b[1;32m---> 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_sparsetools\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (csr_tocsc, csr_tobsr, csr_count_blocks,\n\u001b[0;32m     12\u001b[0m                            get_csr_submatrix)\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_sputils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m upcast\n\u001b[0;32m     15\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_compressed\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _cs_matrix\n",
      "\u001b[1;31mImportError\u001b[0m: numpy.core.multiarray failed to import"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_selection import SelectKBest,f_classif\n",
    "\n",
    "f,p=f_classif(train[num_features],train['y'])\n",
    "k = f.shape[0] - (p > 0.05).sum()\n",
    "selector = SelectKBest(f_classif, k=k)\n",
    "selector.fit(train[num_features],train['y'])\n",
    "\n",
    "print('scores_:',selector.scores_)\n",
    "print('pvalues_:',selector.pvalues_)\n",
    "print('selected index:',selector.get_support(True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "5AC3B3DFF3134BDC83E167086A48540B",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'StandardScaler' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[6], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# 标准化，返回值为标准化后的数据\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m standardScaler\u001b[38;5;241m=\u001b[39mStandardScaler()\n\u001b[0;32m      3\u001b[0m ss\u001b[38;5;241m=\u001b[39mstandardScaler\u001b[38;5;241m.\u001b[39mfit(dummy_train\u001b[38;5;241m.\u001b[39mloc[:,num_features])\n\u001b[0;32m      4\u001b[0m dummy_train\u001b[38;5;241m.\u001b[39mloc[:,num_features]\u001b[38;5;241m=\u001b[39mss\u001b[38;5;241m.\u001b[39mtransform(dummy_train\u001b[38;5;241m.\u001b[39mloc[:,num_features])\n",
      "\u001b[1;31mNameError\u001b[0m: name 'StandardScaler' is not defined"
     ]
    }
   ],
   "source": [
    "# 标准化，返回值为标准化后的数据\n",
    "standardScaler=StandardScaler()\n",
    "ss=standardScaler.fit(dummy_train.loc[:,num_features])\n",
    "dummy_train.loc[:,num_features]=ss.transform(dummy_train.loc[:,num_features])\n",
    "dummy_test.loc[:,num_features]=ss.transform(dummy_test.loc[:,num_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "EEEA48883D2E4FA58FB392991E531C86",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'dummy_train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[7], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m X\u001b[38;5;241m=\u001b[39mdummy_train\n\u001b[0;32m      2\u001b[0m y\u001b[38;5;241m=\u001b[39mtrain[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124my\u001b[39m\u001b[38;5;124m'\u001b[39m]\n",
      "\u001b[1;31mNameError\u001b[0m: name 'dummy_train' is not defined"
     ]
    }
   ],
   "source": [
    "X=dummy_train\n",
    "y=train['y']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "A16ACA2680A243C0880DE1F19CFB6AAD",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "**因为后续是进行回归而非分类，个人认为没有必要进行不平衡处理，故此部分就注释掉了**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "AFE7C8F818B148178A42245BBF4C8878",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# X_train,X_valid,y_train,y_valid=train_test_split(X,y,test_size=0.2,random_state=2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "046EFE395ED34BFCAD11F2721BBE40FD",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# smote_tomek = SMOTETomek(random_state=2020)\n",
    "# X_resampled, y_resampled = smote_tomek.fit_resample(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "13F7B7216054465D9437CEB872026BC3",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "# 数据建模"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "0E1FC19C7DF74042B6F1A9F5527AA4AE",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A module that was compiled using NumPy 1.x cannot be run in\n",
      "NumPy 2.2.6 as it may crash. To support both 1.x and 2.x\n",
      "versions of NumPy, modules must be compiled with NumPy 2.0.\n",
      "Some module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n",
      "\n",
      "If you are a user of the module, the easiest solution will be to\n",
      "downgrade to 'numpy<2' or try to upgrade the affected module.\n",
      "We expect that some modules will need time to support NumPy 2.\n",
      "\n",
      "Traceback (most recent call last):  File \"<frozen runpy>\", line 198, in _run_module_as_main\n",
      "  File \"<frozen runpy>\", line 88, in _run_code\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel_launcher.py\", line 17, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\traitlets\\config\\application.py\", line 1075, in launch_instance\n",
      "    app.start()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelapp.py\", line 701, in start\n",
      "    self.io_loop.start()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\tornado\\platform\\asyncio.py\", line 205, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\windows_events.py\", line 322, in run_forever\n",
      "    super().run_forever()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\base_events.py\", line 641, in run_forever\n",
      "    self._run_once()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\base_events.py\", line 1987, in _run_once\n",
      "    handle._run()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\events.py\", line 88, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in dispatch_queue\n",
      "    await self.process_one()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 523, in process_one\n",
      "    await dispatch(*args)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 429, in dispatch_shell\n",
      "    await result\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 767, in execute_request\n",
      "    reply_content = await reply_content\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\ipkernel.py\", line 429, in do_execute\n",
      "    res = shell.run_cell(\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell\n",
      "    return super().run_cell(*args, **kwargs)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3075, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3130, in _run_cell\n",
      "    result = runner(coro)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3334, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3517, in run_ast_nodes\n",
      "    if await self.run_code(code, result, async_=asy):\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3577, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\zh410st005\\AppData\\Local\\Temp\\ipykernel_10864\\229888457.py\", line 2, in <module>\n",
      "    from sklearn.model_selection import GridSearchCV\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\__init__.py\", line 87, in <module>\n",
      "    from .base import clone\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\base.py\", line 19, in <module>\n",
      "    from .utils import _IS_32BIT\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\utils\\__init__.py\", line 16, in <module>\n",
      "    from scipy.sparse import issparse\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\__init__.py\", line 295, in <module>\n",
      "    from ._csr import *\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\_csr.py\", line 11, in <module>\n",
      "    from ._sparsetools import (csr_tocsc, csr_tobsr, csr_count_blocks,\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "\nA module that was compiled using NumPy 1.x cannot be run in\nNumPy 2.2.6 as it may crash. To support both 1.x and 2.x\nversions of NumPy, modules must be compiled with NumPy 2.0.\nSome module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n\nIf you are a user of the module, the easiest solution will be to\ndowngrade to 'numpy<2' or try to upgrade the affected module.\nWe expect that some modules will need time to support NumPy 2.\n\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python312\\site-packages\\numpy\\core\\_multiarray_umath.py:44\u001b[0m, in \u001b[0;36m__getattr__\u001b[1;34m(attr_name)\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[38;5;66;03m# Also print the message (with traceback).  This is because old versions\u001b[39;00m\n\u001b[0;32m     40\u001b[0m     \u001b[38;5;66;03m# of NumPy unfortunately set up the import to replace (and hide) the\u001b[39;00m\n\u001b[0;32m     41\u001b[0m     \u001b[38;5;66;03m# error.  The traceback shouldn't be needed, but e.g. pytest plugins\u001b[39;00m\n\u001b[0;32m     42\u001b[0m     \u001b[38;5;66;03m# seem to swallow it and we should be failing anyway...\u001b[39;00m\n\u001b[0;32m     43\u001b[0m     sys\u001b[38;5;241m.\u001b[39mstderr\u001b[38;5;241m.\u001b[39mwrite(msg \u001b[38;5;241m+\u001b[39m tb_msg)\n\u001b[1;32m---> 44\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(msg)\n\u001b[0;32m     46\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(_multiarray_umath, attr_name, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m     47\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m ret \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[1;31mImportError\u001b[0m: \nA module that was compiled using NumPy 1.x cannot be run in\nNumPy 2.2.6 as it may crash. To support both 1.x and 2.x\nversions of NumPy, modules must be compiled with NumPy 2.0.\nSome module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n\nIf you are a user of the module, the easiest solution will be to\ndowngrade to 'numpy<2' or try to upgrade the affected module.\nWe expect that some modules will need time to support NumPy 2.\n\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "numpy.core.multiarray failed to import",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[10], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m#----------建模工具--------------#\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msklearn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodel_selection\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m GridSearchCV\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msklearn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpipeline\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Pipeline\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msklearn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodel_selection\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m KFold,RepeatedKFold\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\__init__.py:87\u001b[0m\n\u001b[0;32m     73\u001b[0m     \u001b[38;5;66;03m# We are not importing the rest of scikit-learn during the build\u001b[39;00m\n\u001b[0;32m     74\u001b[0m     \u001b[38;5;66;03m# process, as it may not be compiled yet\u001b[39;00m\n\u001b[0;32m     75\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     81\u001b[0m     \u001b[38;5;66;03m# later is linked to the OpenMP runtime to make it possible to introspect\u001b[39;00m\n\u001b[0;32m     82\u001b[0m     \u001b[38;5;66;03m# it and importing it first would fail if the OpenMP dll cannot be found.\u001b[39;00m\n\u001b[0;32m     83\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     84\u001b[0m         __check_build,  \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[0;32m     85\u001b[0m         _distributor_init,  \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[0;32m     86\u001b[0m     )\n\u001b[1;32m---> 87\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbase\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m clone\n\u001b[0;32m     88\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_show_versions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m show_versions\n\u001b[0;32m     90\u001b[0m     __all__ \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m     91\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcalibration\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     92\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcluster\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    133\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mshow_versions\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m    134\u001b[0m     ]\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\base.py:19\u001b[0m\n\u001b[0;32m     17\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m config_context, get_config\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m InconsistentVersionWarning\n\u001b[1;32m---> 19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _IS_32BIT\n\u001b[0;32m     20\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_estimator_html_repr\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _HTMLDocumentationLinkMixin, estimator_html_repr\n\u001b[0;32m     21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_metadata_requests\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _MetadataRequester, _routing_enabled\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\utils\\__init__.py:16\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mitertools\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m compress, islice\n\u001b[0;32m     15\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m---> 16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msparse\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m issparse\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m get_config\n\u001b[0;32m     19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DataConversionWarning\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\__init__.py:295\u001b[0m\n\u001b[0;32m    292\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwarnings\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01m_warnings\u001b[39;00m\n\u001b[0;32m    294\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_base\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m--> 295\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_csr\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[0;32m    296\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_csc\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[0;32m    297\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_lil\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\_csr.py:11\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_matrix\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m spmatrix\n\u001b[0;32m     10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_base\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _spbase, sparray\n\u001b[1;32m---> 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_sparsetools\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (csr_tocsc, csr_tobsr, csr_count_blocks,\n\u001b[0;32m     12\u001b[0m                            get_csr_submatrix)\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_sputils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m upcast\n\u001b[0;32m     15\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_compressed\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _cs_matrix\n",
      "\u001b[1;31mImportError\u001b[0m: numpy.core.multiarray failed to import"
     ]
    }
   ],
   "source": [
    "#----------建模工具--------------#\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import KFold,RepeatedKFold\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "import xgboost as xgb\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.linear_model import BayesianRidge\n",
    "from catboost import CatBoostRegressor, Pool\n",
    "from lightgbm import LGBMRegressor\n",
    "#----------模型评估工具----------#\n",
    "from sklearn.metrics import confusion_matrix # 混淆矩阵\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import recall_score,f1_score\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import roc_curve,auc\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D59F8B42DDB34650B81C23D56AE0BB43",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 模型建立和参数调整"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "B6C0DDA8DA53482ABA9E2EE753DB105D",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 基于GridSearchCV的随机森林参数调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4451159F5E62465CB52012889AD801B2",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 随机森林\n",
    "# param = {'n_estimators':[1500,1700,2000],\n",
    "#          'max_features':[7,11,15]\n",
    "#         }\n",
    "# gs = GridSearchCV(estimator=RandomForestRegressor(), param_grid=param, cv=3, scoring=\"neg_mean_squared_error\", n_jobs=-1, verbose=10) \n",
    "# gs.fit(X_resampled,y_resampled)\n",
    "# print(gs.best_params_) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "821CD114AA87471B9188366261086B72",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 基于五折交叉验证的随机森林"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "46ED98A1821C450E85F2338B49A197C6",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'KFold' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[11], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m n_fold \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m5\u001b[39m\n\u001b[1;32m----> 2\u001b[0m folds \u001b[38;5;241m=\u001b[39m KFold(n_splits\u001b[38;5;241m=\u001b[39mn_fold, shuffle\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2022\u001b[39m)\n\u001b[0;32m      3\u001b[0m oof_rf \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros(\u001b[38;5;28mlen\u001b[39m(X))\n\u001b[0;32m      4\u001b[0m prediction_rf \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros(\u001b[38;5;28mlen\u001b[39m(dummy_test))\n",
      "\u001b[1;31mNameError\u001b[0m: name 'KFold' is not defined"
     ]
    }
   ],
   "source": [
    "n_fold = 5\n",
    "folds = KFold(n_splits=n_fold, shuffle=True, random_state=2022)\n",
    "oof_rf = np.zeros(len(X))\n",
    "prediction_rf = np.zeros(len(dummy_test))\n",
    "for fold_n, (train_index, valid_index) in enumerate(folds.split(X)):\n",
    "    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n",
    "    y_train, y_valid = y[train_index], y[valid_index]\n",
    "#     smote_tomek = SMOTETomek(random_state=2022)\n",
    "#     X_resampled, y_resampled = smote_tomek.fit_resample(X_train, y_train)\n",
    "    model_rf = RandomForestRegressor(max_features=11,min_samples_leaf=1,n_estimators=1700,random_state=2022).fit(X_train,y_train)\n",
    "    y_pred_valid = model_rf.predict(X_valid)\n",
    "    y_pred = model_rf.predict(dummy_test)\n",
    "    oof_rf[valid_index] = y_pred_valid.reshape(-1, )\n",
    "    prediction_rf += y_pred\n",
    "prediction_rf /= n_fold \n",
    "print(roc_auc_score(y, oof_rf))\n",
    "#0.929373220326099"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "A03808ADDE7A4AC4AEFDC87F09A5A017",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 基于GridSearchCV的XGB参数调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "A26971BEEF684860B739522723029593",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# param = {'max_depth': [3],\n",
    "#          'learning_rate': [0.01],\n",
    "#         'subsample':[0.8],\n",
    "#         'colsample_bytree':[0.6],\n",
    "#          'n_estimators': [8000]\n",
    "\n",
    "#         }\n",
    "# gs = GridSearchCV(estimator=XGBRegressor(), param_grid=param, cv=3, scoring=\"neg_mean_squared_error\", n_jobs=-1, verbose=10) \n",
    "# gs.fit(X,y)\n",
    "# print(gs.best_params_) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7AF59354E891478D8154F963C5EDE251",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 基于五折交叉验证的XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "C0C60825A129420F8F4C983DEC926AC6",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'KFold' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[13], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m n_fold \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m5\u001b[39m\n\u001b[1;32m----> 2\u001b[0m folds \u001b[38;5;241m=\u001b[39m KFold(n_splits\u001b[38;5;241m=\u001b[39mn_fold, shuffle\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2022\u001b[39m)\n\u001b[0;32m      3\u001b[0m oof_xgb \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros(\u001b[38;5;28mlen\u001b[39m(X))\n\u001b[0;32m      4\u001b[0m prediction_xgb \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros(\u001b[38;5;28mlen\u001b[39m(dummy_test))\n",
      "\u001b[1;31mNameError\u001b[0m: name 'KFold' is not defined"
     ]
    }
   ],
   "source": [
    "n_fold = 5\n",
    "folds = KFold(n_splits=n_fold, shuffle=True, random_state=2022)\n",
    "oof_xgb = np.zeros(len(X))\n",
    "prediction_xgb = np.zeros(len(dummy_test))\n",
    "for fold_n, (train_index, valid_index) in enumerate(folds.split(X)):\n",
    "    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n",
    "    y_train, y_valid = y[train_index], y[valid_index]\n",
    "#     smote_tomek = SMOTETomek(random_state=2022)\n",
    "#     X_resampled, y_resampled = smote_tomek.fit_resample(X_train, y_train)\n",
    "    eval_set = [(X_valid, y_valid)]\n",
    "    model_xgb = XGBRegressor(\n",
    "        max_depth=9,learning_rate=0.01,n_estimators=10000,colsample_bytree=0.6,subsample=0.8,random_state=2022\n",
    "    ).fit(X_train,y_train,early_stopping_rounds=100, eval_metric=\"auc\",eval_set=eval_set, verbose=True)\n",
    "    y_pred_valid = model_xgb.predict(X_valid)\n",
    "    y_pred = model_xgb.predict(dummy_test)\n",
    "    oof_xgb[valid_index] = y_pred_valid.reshape(-1, )\n",
    "    prediction_xgb += y_pred\n",
    "prediction_xgb /= n_fold \n",
    "print(roc_auc_score(y, oof_xgb))\n",
    "# 0.9326219985474677"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D03CB6619D8B44B08E990ED3EA01C61D",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 基于GridSearchCV的LGBM参数调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "082D0D1C79D14C0BB07870203B74F5F8",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# param = {'max_depth': [30],\n",
    "#          'learning_rate': [0.01],\n",
    "#          'num_leaves': [59],\n",
    "#          'subsample': [0.7],\n",
    "#          'colsample_bytree': [0.8],\n",
    "#          'n_estimators': [10000]}\n",
    "# gs = GridSearchCV(estimator=LGBMRegressor(), param_grid=param, cv=5, scoring=\"neg_mean_squared_error\", n_jobs=-1) \n",
    "# gs.fit(X_resampled,y_resampled)\n",
    "# print(gs.best_params_) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "23ADF0543B77452A8DCC799C0582D474",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 基于五折交叉验证的LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "D2313DA33FCA4D7EA5F3C32DD98D6BBA",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'KFold' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[15], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m n_fold \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m5\u001b[39m\n\u001b[1;32m----> 2\u001b[0m folds \u001b[38;5;241m=\u001b[39m KFold(n_splits\u001b[38;5;241m=\u001b[39mn_fold, shuffle\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1314\u001b[39m)\n\u001b[0;32m      3\u001b[0m params \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m      4\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlearning_rate\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;241m0.01\u001b[39m,\n\u001b[0;32m      5\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msubsample\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;241m0.7\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mn_jobs\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m\n\u001b[0;32m     13\u001b[0m }\n\u001b[0;32m     15\u001b[0m oof_lgb \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros(\u001b[38;5;28mlen\u001b[39m(X))\n",
      "\u001b[1;31mNameError\u001b[0m: name 'KFold' is not defined"
     ]
    }
   ],
   "source": [
    "n_fold = 5\n",
    "folds = KFold(n_splits=n_fold, shuffle=True,random_state=1314)\n",
    "params = {\n",
    "    'learning_rate':0.01,\n",
    "    'subsample': 0.7,\n",
    "    'num_leaves': 59,\n",
    "    'n_estimators':1500,\n",
    "    'max_depth': 30,\n",
    "    'colsample_bytree': 0.8,\n",
    "    'verbose': -1,\n",
    "    'seed': 2022,\n",
    "    'n_jobs': -1\n",
    "}\n",
    "\n",
    "oof_lgb = np.zeros(len(X))\n",
    "predictions_lgb  = np.zeros(len(dummy_test))\n",
    "for fold_n, (train_index, valid_index) in enumerate(folds.split(X)):\n",
    "    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n",
    "    y_train, y_valid = y[train_index], y[valid_index]\n",
    "#     smote_tomek = SMOTETomek(random_state=2022)\n",
    "#     X_resampled, y_resampled = smote_tomek.fit_resample(X_train, y_train)\n",
    "    model = lgb.LGBMRegressor(**params)\n",
    "    model.fit(X_train, y_train,\n",
    "              eval_set=[(X_train, y_train), (X_valid, y_valid)],\n",
    "              eval_metric='auc',\n",
    "              verbose=50, early_stopping_rounds=200)\n",
    "    y_pred_valid = model.predict(X_valid)\n",
    "    y_pred = model.predict(dummy_test, num_iteration=model.best_iteration_)\n",
    "    oof_lgb[valid_index] = y_pred_valid.reshape(-1, )\n",
    "    predictions_lgb  += y_pred\n",
    "predictions_lgb  /= n_fold\n",
    "print(roc_auc_score(y, oof_lgb))\n",
    "# 0.9342991211145983"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1D0CF99ED0F04505ADAF99D733B04257",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 基于GridSearchCV的catboost参数调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "40143CE0A1F7461A9D5AAFEB614609B5",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "# param = {'depth': [7,9,11],\n",
    "#          'learning_rate': [0.01],\n",
    "#          'iterations':  [8000]}\n",
    "# gs = GridSearchCV(estimator=CatBoostRegressor(), param_grid=param, cv=3, scoring=\"neg_mean_squared_error\", n_jobs=-1) \n",
    "# gs.fit(X_resampled,y_resampled)\n",
    "# print(gs.best_params_) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0E595CF0D95341E69AB1154B1849F14E",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 基于五折交叉验证的catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "3A557EC583C0481488702DC367D10F53",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'KFold' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[17], line 3\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# 本地交叉验证\u001b[39;00m\n\u001b[0;32m      2\u001b[0m n_fold \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m5\u001b[39m\n\u001b[1;32m----> 3\u001b[0m folds \u001b[38;5;241m=\u001b[39m KFold(n_splits\u001b[38;5;241m=\u001b[39mn_fold, shuffle\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1314\u001b[39m)\n\u001b[0;32m      5\u001b[0m oof_cat \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros(\u001b[38;5;28mlen\u001b[39m(X))\n\u001b[0;32m      6\u001b[0m prediction_cat \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros(\u001b[38;5;28mlen\u001b[39m(dummy_test))\n",
      "\u001b[1;31mNameError\u001b[0m: name 'KFold' is not defined"
     ]
    }
   ],
   "source": [
    "# 本地交叉验证\n",
    "n_fold = 5\n",
    "folds = KFold(n_splits=n_fold, shuffle=True, random_state=1314)\n",
    "\n",
    "oof_cat = np.zeros(len(X))\n",
    "prediction_cat = np.zeros(len(dummy_test))\n",
    "for fold_n, (train_index, valid_index) in enumerate(folds.split(X)):\n",
    "    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n",
    "    y_train, y_valid = y[train_index], y[valid_index]\n",
    "#     smote_tomek = SMOTETomek(random_state=2022)\n",
    "#     X_resampled, y_resampled = smote_tomek.fit_resample(X_train, y_train)\n",
    "    train_pool = Pool(X_train, y_train)\n",
    "    eval_pool = Pool(X_valid, y_valid)\n",
    "    cbt_model = CatBoostRegressor(iterations=25000, # 注：baseline 提到的分数是用 iterations=60000 得到的，但运行时间有点久\n",
    "                           learning_rate=0.01, # 注：事实上好几个 property 在 lr=0.1 时收敛巨慢。后面可以考虑调大\n",
    "#                            eval_metric='SMAPE',\n",
    "                                  depth=9,\n",
    "                           use_best_model=True,\n",
    "                           random_seed=2022,\n",
    "                           logging_level='Verbose',\n",
    "                           #task_type='GPU',\n",
    "                           devices='0',\n",
    "                           gpu_ram_part=0.5,\n",
    "                           early_stopping_rounds=300)\n",
    "    \n",
    "    cbt_model.fit(train_pool,\n",
    "              eval_set=eval_pool,\n",
    "              verbose=1000)\n",
    "\n",
    "    y_pred_valid = cbt_model.predict(X_valid)\n",
    "    y_pred_c = cbt_model.predict(dummy_test)\n",
    "    oof_cat[valid_index] = y_pred_valid.reshape(-1, )\n",
    "    prediction_cat += y_pred_c\n",
    "prediction_cat /= n_fold \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "27A63B61DB094FCF95CEEB6A1E3ADC5A",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'roc_auc_score' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[18], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28mprint\u001b[39m(roc_auc_score(y, oof_cat))\n",
      "\u001b[1;31mNameError\u001b[0m: name 'roc_auc_score' is not defined"
     ]
    }
   ],
   "source": [
    "print(roc_auc_score(y, oof_cat))\n",
    "# 0.935264298588153"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9470A77D276E418F9CB2ACD8D18E6F68",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 基于stacking的模型融合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "4A24F209D61E47E7A27F4A26D6687B89",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A module that was compiled using NumPy 1.x cannot be run in\n",
      "NumPy 2.2.6 as it may crash. To support both 1.x and 2.x\n",
      "versions of NumPy, modules must be compiled with NumPy 2.0.\n",
      "Some module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n",
      "\n",
      "If you are a user of the module, the easiest solution will be to\n",
      "downgrade to 'numpy<2' or try to upgrade the affected module.\n",
      "We expect that some modules will need time to support NumPy 2.\n",
      "\n",
      "Traceback (most recent call last):  File \"<frozen runpy>\", line 198, in _run_module_as_main\n",
      "  File \"<frozen runpy>\", line 88, in _run_code\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel_launcher.py\", line 17, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\traitlets\\config\\application.py\", line 1075, in launch_instance\n",
      "    app.start()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelapp.py\", line 701, in start\n",
      "    self.io_loop.start()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\tornado\\platform\\asyncio.py\", line 205, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\windows_events.py\", line 322, in run_forever\n",
      "    super().run_forever()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\base_events.py\", line 641, in run_forever\n",
      "    self._run_once()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\base_events.py\", line 1987, in _run_once\n",
      "    handle._run()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\asyncio\\events.py\", line 88, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in dispatch_queue\n",
      "    await self.process_one()\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 523, in process_one\n",
      "    await dispatch(*args)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 429, in dispatch_shell\n",
      "    await result\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 767, in execute_request\n",
      "    reply_content = await reply_content\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\ipkernel.py\", line 429, in do_execute\n",
      "    res = shell.run_cell(\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell\n",
      "    return super().run_cell(*args, **kwargs)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3075, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3130, in _run_cell\n",
      "    result = runner(coro)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3334, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3517, in run_ast_nodes\n",
      "    if await self.run_code(code, result, async_=asy):\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3577, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\zh410st005\\AppData\\Local\\Temp\\ipykernel_10864\\1898858170.py\", line 2, in <module>\n",
      "    from sklearn.metrics import mean_squared_error,mean_absolute_error,make_scorer\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\__init__.py\", line 87, in <module>\n",
      "    from .base import clone\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\base.py\", line 19, in <module>\n",
      "    from .utils import _IS_32BIT\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\utils\\__init__.py\", line 16, in <module>\n",
      "    from scipy.sparse import issparse\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\__init__.py\", line 295, in <module>\n",
      "    from ._csr import *\n",
      "  File \"D:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\_csr.py\", line 11, in <module>\n",
      "    from ._sparsetools import (csr_tocsc, csr_tobsr, csr_count_blocks,\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "\nA module that was compiled using NumPy 1.x cannot be run in\nNumPy 2.2.6 as it may crash. To support both 1.x and 2.x\nversions of NumPy, modules must be compiled with NumPy 2.0.\nSome module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n\nIf you are a user of the module, the easiest solution will be to\ndowngrade to 'numpy<2' or try to upgrade the affected module.\nWe expect that some modules will need time to support NumPy 2.\n\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python312\\site-packages\\numpy\\core\\_multiarray_umath.py:44\u001b[0m, in \u001b[0;36m__getattr__\u001b[1;34m(attr_name)\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[38;5;66;03m# Also print the message (with traceback).  This is because old versions\u001b[39;00m\n\u001b[0;32m     40\u001b[0m     \u001b[38;5;66;03m# of NumPy unfortunately set up the import to replace (and hide) the\u001b[39;00m\n\u001b[0;32m     41\u001b[0m     \u001b[38;5;66;03m# error.  The traceback shouldn't be needed, but e.g. pytest plugins\u001b[39;00m\n\u001b[0;32m     42\u001b[0m     \u001b[38;5;66;03m# seem to swallow it and we should be failing anyway...\u001b[39;00m\n\u001b[0;32m     43\u001b[0m     sys\u001b[38;5;241m.\u001b[39mstderr\u001b[38;5;241m.\u001b[39mwrite(msg \u001b[38;5;241m+\u001b[39m tb_msg)\n\u001b[1;32m---> 44\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(msg)\n\u001b[0;32m     46\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(_multiarray_umath, attr_name, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m     47\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m ret \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[1;31mImportError\u001b[0m: \nA module that was compiled using NumPy 1.x cannot be run in\nNumPy 2.2.6 as it may crash. To support both 1.x and 2.x\nversions of NumPy, modules must be compiled with NumPy 2.0.\nSome module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n\nIf you are a user of the module, the easiest solution will be to\ndowngrade to 'numpy<2' or try to upgrade the affected module.\nWe expect that some modules will need time to support NumPy 2.\n\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "numpy.core.multiarray failed to import",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[19], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# from sklearn.linear_model import Bayesian\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msklearn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m mean_squared_error,mean_absolute_error,make_scorer\n\u001b[0;32m      4\u001b[0m \u001b[38;5;66;03m# 将多个模型的结果进行stacking（叠加）\u001b[39;00m\n\u001b[0;32m      5\u001b[0m train_stack \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mvstack([oof_rf,oof_lgb,oof_cat,oof_xgb])\u001b[38;5;241m.\u001b[39mtranspose()\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\__init__.py:87\u001b[0m\n\u001b[0;32m     73\u001b[0m     \u001b[38;5;66;03m# We are not importing the rest of scikit-learn during the build\u001b[39;00m\n\u001b[0;32m     74\u001b[0m     \u001b[38;5;66;03m# process, as it may not be compiled yet\u001b[39;00m\n\u001b[0;32m     75\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     81\u001b[0m     \u001b[38;5;66;03m# later is linked to the OpenMP runtime to make it possible to introspect\u001b[39;00m\n\u001b[0;32m     82\u001b[0m     \u001b[38;5;66;03m# it and importing it first would fail if the OpenMP dll cannot be found.\u001b[39;00m\n\u001b[0;32m     83\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     84\u001b[0m         __check_build,  \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[0;32m     85\u001b[0m         _distributor_init,  \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[0;32m     86\u001b[0m     )\n\u001b[1;32m---> 87\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbase\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m clone\n\u001b[0;32m     88\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_show_versions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m show_versions\n\u001b[0;32m     90\u001b[0m     __all__ \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m     91\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcalibration\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     92\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcluster\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    133\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mshow_versions\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m    134\u001b[0m     ]\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\base.py:19\u001b[0m\n\u001b[0;32m     17\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m config_context, get_config\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m InconsistentVersionWarning\n\u001b[1;32m---> 19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _IS_32BIT\n\u001b[0;32m     20\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_estimator_html_repr\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _HTMLDocumentationLinkMixin, estimator_html_repr\n\u001b[0;32m     21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_metadata_requests\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _MetadataRequester, _routing_enabled\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\utils\\__init__.py:16\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mitertools\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m compress, islice\n\u001b[0;32m     15\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m---> 16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msparse\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m issparse\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m get_config\n\u001b[0;32m     19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DataConversionWarning\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\__init__.py:295\u001b[0m\n\u001b[0;32m    292\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwarnings\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01m_warnings\u001b[39;00m\n\u001b[0;32m    294\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_base\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m--> 295\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_csr\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[0;32m    296\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_csc\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[0;32m    297\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_lil\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\scipy\\sparse\\_csr.py:11\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_matrix\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m spmatrix\n\u001b[0;32m     10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_base\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _spbase, sparray\n\u001b[1;32m---> 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_sparsetools\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (csr_tocsc, csr_tobsr, csr_count_blocks,\n\u001b[0;32m     12\u001b[0m                            get_csr_submatrix)\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_sputils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m upcast\n\u001b[0;32m     15\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_compressed\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _cs_matrix\n",
      "\u001b[1;31mImportError\u001b[0m: numpy.core.multiarray failed to import"
     ]
    }
   ],
   "source": [
    "# from sklearn.linear_model import Bayesian\n",
    "from sklearn.metrics import mean_squared_error,mean_absolute_error,make_scorer\n",
    "\n",
    "# 将多个模型的结果进行stacking（叠加）\n",
    "train_stack = np.vstack([oof_rf,oof_lgb,oof_cat,oof_xgb]).transpose()\n",
    "test_stack = np.vstack([prediction_rf,prediction_lgb,prediction_cat,prediction_xgb]).transpose()\n",
    "#贝叶斯分类器也使用交叉验证的方法，5折，重复2次\n",
    "folds_stack = RepeatedKFold(n_splits=5, n_repeats=2, random_state=2018)\n",
    "oof_stack = np.zeros(train_stack.shape[0])\n",
    "predictions = np.zeros(test_stack.shape[0])\n",
    " \n",
    "for fold_, (trn_idx, val_idx) in enumerate(folds_stack.split(train_stack,y)):\n",
    "    print(\"fold {}\".format(fold_))\n",
    "    trn_data, trn_y = train_stack[trn_idx], y.iloc[trn_idx].values\n",
    "    val_data, val_y = train_stack[val_idx], y.iloc[val_idx].values#\n",
    "    \n",
    "    clf_3 = BayesianRidge()\n",
    "    clf_3.fit(trn_data, trn_y)\n",
    "    \n",
    "    oof_stack[val_idx] = clf_3.predict(val_data)#对验证集有一个预测，用于后面计算模型的偏差\n",
    "    predictions += clf_3.predict(test_stack) / 10#对测试集的预测，除以10是因为5折交叉验证重复了2次\n",
    "    \n",
    "mean_squared_error(y.values, oof_stack)#计算出模型在训练集上的均方误差\n",
    "print(\"CV score: {:<8.8f}\".format(mean_squared_error(y.values, oof_stack)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "33AB2F850236462DB7F0DFE549A2C559",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(roc_auc_score(y, oof_stack))\n",
    "# 0.9361018703876826"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1D7C809913E14DD79D1E399F519EDC1C",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "# 保存结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "BC37DA49289D45C2951D2FFDC91043BA",
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'predictions' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[20], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m test[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpred\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m predictions\n\u001b[0;32m      2\u001b[0m test[[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mID\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpred\u001b[39m\u001b[38;5;124m'\u001b[39m]]\u001b[38;5;241m.\u001b[39mto_csv(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mC:\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mUsers\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mhepei\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mDesktop\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124m比赛代码\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124m练习赛\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124m客户购买预测\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124m结果\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124msub.csv\u001b[39m\u001b[38;5;124m'\u001b[39m, index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, encoding\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'predictions' is not defined"
     ]
    }
   ],
   "source": [
    "test['pred'] = predictions\n",
    "test[['ID', 'pred']].to_csv(r'C:\\Users\\hepei\\Desktop\\比赛代码\\练习赛\\客户购买预测\\结果\\sub.csv', index=None, encoding=\"utf-8\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
